import Queue
import socket
from threading import Thread,RLock,stack_size
import urllib
import urllib2
import time

class Fetcher(object):
    """

    """
    def __init__(self, threads_number=5, stacksize=32768*16):
        """
        Constructor for Fetcher
        """
        self.q_request = Queue.Queue()
        self.q_answer = Queue.Queue()
        self.lock = RLock()
        self.running = 0
        #self.opener = urllib2.build_opener(urllib2.HTTPHandler)
        self.login()
        stack_size(stacksize)
        for i in range(threads_number):
            t = Thread(target=self.thread_work)
            t.setDaemon(True)
            t.start()

    def __del__(self):
        time.sleep(0.5)
        self.q_request.join()
        self.q_answer.join()

    def login(self,data='',login_url=''):
        #return self.opener.open(login_url,urllib.urlencode(data)).read()
        """
        如果子类需要登录网站，则重写该函数
        """
        pass
    
    def task_left(self):
        """
        返回剩余任务数，包括请求任务，完成任务，和正在运行的任务
        """
        return self.q_request.qsize()+self.q_answer.qsize()+self.running

    def push(self,req):
        self.q_request.put(req)

    def pop(self):
        return self.q_answer.get()
    
    def thread_work(self):
        while True:
            req = self.q_request.get()
            with self.lock:
                self.running += 1
            try:
                ans = self.working(req)
            except Exception,e:
                ans = ''
                print e
            self.q_answer.put((req,ans))
            with self.lock:
                self.running -= 1
            self.q_request.task_done()
            time.sleep(0.1)

    def working(self,req):
        #return self.opener.open(req).read()
        pass

    def set_timeout(self,timeout):
        """
        设置连接超时时间
        """
        socket.setdefaulttimeout(timeout)

                
#if __name__ == "__main__":
#    links = [ 'http://www.verycd.com/topics/%d/'%i for i in range(5420,5430) ]
#    f = Fetcher(threads=10)
#    for url in links:
#        f.push(url)
#    while f.taskleft():
#        url,content = f.pop()
#        print url,len(content)